package fi.foyt.foursquare.api;

import java.io.*;
import java.util.*;

import fi.foyt.foursquare.api.*;
import fi.foyt.foursquare.api.io.*;
import fi.foyt.foursquare.api.entities.*;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;

import org.w3c.dom.Attr;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import javax.xml.stream.*;

public class CrawlerExample {
	
	private static String clientID = "Put here your FourSquare Client ID";
	private static String clientSecret = "Put here your FourSquare Client Secret";
	private static String callbackURL = "Put here your FourSquare Callback URL";
	private static Integer radius = 1000;
	private static String intent = "browse";
	private static Integer limit = 50;
	private static String oAuthToken = "Put here your FourSquare oAuthToken";
	
	
	/* To determine whether API method venuesSearch returned an error*/
	private static boolean errorSearchVenues = false;
	
	/* To determine whether API method venue returned an error*/
	private static boolean errorVenue = false;		

	/* To determine whether a JSON Exception was caught*/
	private static boolean jsonExceptionFound = false;
	
	/* To store every searched venue (alpha-numeric string)	*/
	private static HashSet<String> checkedVenues = new HashSet<String>(); 
	
	/* To store every user checked (userId)*/
	private static HashSet<Integer> checkedUsers = new HashSet<Integer>(); 
	
	
	public static void main ( String args[] ) throws Exception {
		
		/*Creating XML file with FourSquare data*/
		XMLOutputFactory xmlof = XMLOutputFactory.newInstance();
		XMLStreamWriter xmlw = xmlof.createXMLStreamWriter(new FileOutputStream ("fourSquareData.xml"));
		
		// Write the default XML declaration
		xmlw.writeStartDocument("UTF-8",""); 
		//line-breaks are not "automatic"
		xmlw.writeCharacters("\n"); 
		
		//the root element of this xml file
		xmlw.writeStartElement("data");
		xmlw.writeCharacters("\n");
		
		
		/* Searching for venues in the NYC box 
		*
		*	Latitude : 40.495682 to 40.917622
		*	Longitude : -74.255653 to -73.689484 
		*
		*/
		for ( double lat=40.495682; lat < 40.917622; lat += 0.001 ){
			for (double lon=-74.255653; lon < -73.689484; lon += 0.001 ){
				
				if(errorSearchVenues){					
					lon -= 0.001;
					errorSearchVenues = false;
					Thread.sleep(60000);
					searchVenues(lat+","+lon, xmlw);
				}else{
					searchVenues(lat+","+lon,xmlw);
					Thread.sleep(1000);
				}
			}
		}
		
		xmlw.writeEndElement(); //close the <data> XML Tag
		xmlw.writeCharacters("\n"); 
		xmlw.writeEndDocument(); //close the XML Document
		
	}		
	
	public static void searchVenues(String ll,XMLStreamWriter xmlw) throws FoursquareApiException,IOException,InterruptedException, XMLStreamException{
		FoursquareApi foursquareApi = new FoursquareApi(clientID, clientSecret, callbackURL, oAuthToken, new DefaultIOHandler());
		
		/* @param ll latitude and longitude of the user's location. (Required for query searches)
		* @param llAcc accuracy of latitude and longitude, in meters. (Does not currently affect search results.)
		* @param alt altitude of the user's location, in meters. (Does not currently affect search results.)
		* @param altAcc accuracy of the user's altitude, in meters. (Does not currently affect search results.)
		* @param query a search term to be applied against titles.
		* @param limit number of results to return, up to 50.
		* @param intent one of checkin, match or specials
		* @param categoryId a category to limit results to
		* @param url a third-party URL
		* @param providerId identifier for a known third party
		* @param linkedId identifier used by third party specifed in providerId parameter
		* @param radius limit results to venues within this many meters of the specified location
		*/
		
		Result<VenuesSearchResult> result = null;

		try{
			result = foursquareApi.venuesSearch(ll, null, null, null, null,limit,intent, null, null, null, null,radius);				
		}catch(FoursquareApiException e){
			jsonExceptionFound = true;
		}

		if(jsonExceptionFound == false){
		/* To store all users that visited the venue */
		HashSet<CompactUser> userSet = new HashSet<CompactUser>();		
		
		if (result.getMeta().getCode() == 200) {
			
			System.out.println("Venue search for " + ll + " with code = 200");
			
			HashSet<CompactVenue> venueSet = new HashSet<CompactVenue>();
			
			for ( CompactVenue compactVenue : result.getResult().getVenues() ) venueSet.add(compactVenue);
			
			/* Retrieving venue information */
			CompactVenue[] venueArray= (CompactVenue []) venueSet.toArray(new CompactVenue[venueSet.size()]);
			
			for(int j = 0; j < venueArray.length; j++){
				errorVenue = false;
				
				if(!checkedVenues.contains(venueArray[j].getId())) {
					
					Result<CompleteVenue> venueResult = foursquareApi.venue(venueArray[j].getId());
					
					if(venueResult.getMeta().getCode() == 200){
						
						CompleteVenue venue = venueResult.getResult();				
						if(venue != null){							
							
							checkedVenues.add(venue.getId());																
							System.out.println("Venue " + venue.getName() + ":" + venue.getId() + ":" + venue.getLocation().getLat() + "," + venue.getLocation().getLng());
							
							xmlw.writeStartElement("venue"); 
							
							xmlw.writeAttribute("name","" + venue.getName());
							xmlw.writeAttribute("id","" + venue.getId()); 
							xmlw.writeAttribute("lat","" + venue.getLocation().getLat());
							xmlw.writeAttribute("lon","" + venue.getLocation().getLng());
							xmlw.writeCharacters("\n");
							
							/* Retrieving and writing all categories */
							xmlw.writeStartElement("categories");
							for ( Category category : venue.getCategories() ){							
								xmlw.writeCharacters("\n");
								xmlw.writeStartElement("category");
								xmlw.writeCharacters("" + category.getName());
								xmlw.writeEndElement(); //close the <category> XML Tag
							}					
							xmlw.writeCharacters("\n");
							xmlw.writeEndElement(); //close the <categories> XML Tag
							xmlw.writeCharacters("\n");
							
							/* Retrieving and writing venue tags */
							xmlw.writeStartElement("tags");
							for ( String tag : venue.getTags() ){							
								xmlw.writeCharacters("\n");
								xmlw.writeStartElement("tag");
								xmlw.writeCharacters("" + tag);
								xmlw.writeEndElement(); //close the <tag> XML Tag
							}									
							xmlw.writeCharacters("\n");
							xmlw.writeEndElement(); //close the <tags> XML Tag
							xmlw.writeCharacters("\n");
							
							
							
							/* Retrieving all users from this venue */					
							PhotoGroup[] venuePhotoGroup = venue.getPhotos().getGroups();
							
							for(PhotoGroup pg : venuePhotoGroup){
								Photo[] venuePhotos = pg.getItems();
								/* Collectting all users that are associated with photos */
								for(Photo p : venuePhotos){
									userSet.add(p.getUser());
								}
							}													
							
							/* Retrieving the mayor of the venue */	
							CompactUser mayor = venue.getMayor().getUser();
							
							userSet.add(mayor);				
							
							/* Retrieving and writing all users that visitied this venue */
							Iterator it = userSet.iterator();					
							
							/* So that there are no repeated visitors */
							HashSet<Integer> visitorSet = new HashSet<Integer>();
							
							xmlw.writeStartElement("visitingUsers");
							
							while(it.hasNext()){
								
								CompactUser cUser = (CompactUser) it.next();
								
								if(cUser != null){
									
									int u = Integer.parseInt(cUser.getId());
									
									if(!visitorSet.contains(u)){
										
										visitorSet.add(u);
										
										xmlw.writeCharacters("\n");
										xmlw.writeStartElement("visitor");
										xmlw.writeCharacters("" + u);					
										xmlw.writeEndElement(); //close the <visitor> XML Tag
									}
								}
							}							
							xmlw.writeCharacters("\n");
							xmlw.writeEndElement(); //close the <visitingUsers> XML Tag
							xmlw.writeCharacters("\n");														
							
							System.out.println("Finished writing <categories>, <tags> and <visitingUsers>");
							
							System.out.println("Retrieving User Information");
							
							/* Retrieving user information */					
							CompactUser[] userArray= (CompactUser []) userSet.toArray(new CompactUser[userSet.size()]);
							
							for(int i = 0 ; i < userArray.length ; i++){
								
								CompactUser user = userArray[i];
								
								if(user != null){
									
									int userId = Integer.parseInt(user.getId());
									
									if(!checkedUsers.contains(userId)){																												
										
										Result<UserGroup> resultFriends = foursquareApi.usersFriends("" + userId);															
										
										if (resultFriends.getMeta().getCode() == 200) {
											
											/* This user is added and the information is written if there is no error gathering user friends */
											checkedUsers.add(userId);								
											
											System.out.println("user:" + userId + ":" + user.getFirstName() + " " + user.getLastName() + ":" + user.getGender() + ":" + user.getHomeCity());																
											
											/* Writing user information*/
											xmlw.writeStartElement("user");
											xmlw.writeAttribute("id","" + userId);
											xmlw.writeAttribute("fName","" + user.getFirstName());
											xmlw.writeAttribute("lName","" + user.getLastName());
											xmlw.writeAttribute("gender","" + user.getGender());
											xmlw.writeAttribute("hometown","" + user.getHomeCity());								
											xmlw.writeCharacters("\n");
											
											
											UserGroup friendsGroup = resultFriends.getResult();
											
											CompactUser [] friends = friendsGroup.getItems();							
											
											/* Storing all the ids from this user's friends*/								
											
											xmlw.writeStartElement("userFriends");
											for ( CompactUser friend : friends) {									
												int friendId = Integer.parseInt(friend.getId());																		
												
												xmlw.writeCharacters("\n");
												xmlw.writeStartElement("friend");
												xmlw.writeCharacters("" + friendId);					
												xmlw.writeEndElement(); //close the <friend> XML Tag	
											}																								
											xmlw.writeCharacters("\n");
											xmlw.writeEndElement(); //closing <user> XML tag
											xmlw.writeCharacters("\n");
											xmlw.writeEndElement(); //close the <userFriends> XML Tag
											xmlw.writeCharacters("\n");
											
											System.out.println("Finished writing user's " + userId + " friends");
											
										} else {
											
											System.err.println("<UserFriends> Error occured: ");
											
											int code = resultFriends.getMeta().getCode();
											String type = resultFriends.getMeta().getErrorType();
											String detail = resultFriends.getMeta().getErrorDetail();
											
											System.err.println("  code: " + code);
											System.err.println("  type: " + type);
											System.err.println("  detail: " + detail);
											
											System.out.println("Error in " + userId + " <UserFriends> code: " + code + " type: " + type + " detail: " + detail);
											
											/* if the http connection time expires (code = 500) we want to move on */
											if(code == 500){
												continue;
											}else{
												/* if we get an network error (eg, 502) or if we have already exceeded the hourly
												*  limit for API requests we just want to stop for a bit and try again
												*/
												if((code == 403 && type.equals("rate_limit_exceeded")) || (code > 500 && code < 600)){								
													Thread.sleep(60000);
													i--;
													continue;
												}
											}											
										}

										System.out.println("*********\n");
						} else {continue;}												
								}
							}
				} else {continue;}
				
					}else{
						
						System.err.println("<Venue> Error occured: ");
						
						int code = venueResult.getMeta().getCode();
						String type = venueResult.getMeta().getErrorType();
						String detail = venueResult.getMeta().getErrorDetail();
						
						System.err.println("  code: " + code);
						System.err.println("  type: " + type);
						System.err.println("  detail: " + detail);
						
						System.out.println("Error in " + venueArray[j].getId() + " <Venue> code: " + code + " type: " + type + " detail: " + detail);
						
						/* if the http connection time expires (code = 500) we want to move on */
						if(code == 500){
							continue;
						}else{
								/* if we get an network error (eg, 502) or if we have already exceeded the hourly
								*  limit for API requests we just want to stop for a bit and try again
								*/
								if((code == 403 && type.equals("rate_limit_exceeded")) || (code > 500 && code < 600)){
									errorVenue = true;				
									Thread.sleep(60000);
									j--;
									continue;
								}
							}
					}
					
					}else{continue;}
					
					if(errorVenue == false){
						xmlw.writeEndElement(); //close the <venue> XML Tag
						xmlw.writeCharacters("\n");
					}
			}			
		} else {				
			System.err.println("<VenuesSearch> Error occured: ");
			
			int code = result.getMeta().getCode();
			String type = result.getMeta().getErrorType();
			String detail = result.getMeta().getErrorDetail();
			
			System.err.println("  code: " + code);
			System.err.println("  type: " + type);
			System.err.println("  detail: " + detail);
			
			System.out.println("Error in " + ll + " <VenuesSearch> code: " + code + " type: " + type + " detail: " + detail);
			
			/* if the http connection time expires (code = 500) we want to nothing so, there is no need to "isolate" that condition
			*  if we get an network error (eg, 502) or if we have already exceeded the hourly
			*  limit for API requests we just want to stop for a bit and try again
			*/
			if((code == 403 && type.equals("rate_limit_exceeded")) || (code > 500 && code < 600)){								
				errorSearchVenues = true;				
			}
				
		}
	}else{
		jsonExceptionFound = false;
		System.err.println("<VenuesSearch> JSONException found");
		System.out.println("<VenuesSearch> JSONException found");
	}
}
}
